import datetime
import copy
import pymongo
import Core.Gadget as Gadget
import Core.Algorithm as Algo
import Core.IO as IO
from Factors.General import *
import Core.DataSeries as DataSeries

# dependentFinancialName = "OperatingProfit1_LYR"
def SimpleValuationCallBack(dfCombine, factorName, params={}):
    # Example
    # dfCombine["PriceBookLF"] = dfCombine["Cap"] / dfCombine["TotalEquity"]
    # ---Calculation LYR  (Static) Factor---
    # dfCombine["PriceEarningOperatingProfitLYR"] = dfCombine["Cap"] / dfCombine["OperatingProfit1_LYR"]
    # ---Calculation TTM (Rolling) Factor---
    # dfCombine["PriceEarningOperatingProfitTTM"] = dfCombine["Cap"] / dfCombine["OperatingProfit1_TTM"]

    type = ""
    if "Type" in params:
        type = "_" + params["Type"]

    dependentFinancialName = params["Value"]

    r = False
    if "Reciprocal" in params:
        r = params["Reciprocal"]
    #
    if r:
        dfCombine[factorName] = dfCombine[dependentFinancialName + type] / dfCombine["Cap"]
    else:
        dfCombine[factorName] = dfCombine["Cap"] / dfCombine[dependentFinancialName + type]
    #
    return dfCombine


def SimpleValuationReverseCallBack(dfCombine, factorName, params={}):
    #
    type = ""
    if "Type" in params:
        type = "_" + params["Type"]
    dependentFinancialName = params["Value"]
    #
    dfCombine[factorName] = dfCombine[dependentFinancialName + type] / dfCombine["Cap"]
    return dfCombine

# EBIT 用 Opt Profit替代
def EBITTEVCallBack(dfCombine, factorName, params={}):

    #
    # print(dfCombine.head())
    #
    type = params["Type"]

    r = False
    if "Reciprocal" in params:
        r = params["Reciprocal"]

    dfCombine["Cap_Report"] = dfCombine["Close_Report"] * dfCombine["TotalShares_Report"]
    dfCombine["Cap_Report_LYR"] = dfCombine["Close_Report_LYR"] * dfCombine["TotalShares_Report_LYR"]

    # 倒数
    if not r:
        if type == "TTM":
            dfCombine[factorName] = (dfCombine["Cap_Report"] - dfCombine["CashEquivalents"] + dfCombine["TotalLiab"]) / dfCombine["OperatingProfit_TTM"]
        elif type == "LYR":
            dfCombine[factorName] = (dfCombine["Cap_Report_LYR"] - dfCombine["CashEquivalents_LYR"] + dfCombine["TotalLiab_LYR"]) / dfCombine["OperatingProfit_LYR"]
    else:
        #
        if type == "TTM":
            dfCombine[factorName] = dfCombine["OperatingProfit_TTM"] / (dfCombine["Cap_Report"] - dfCombine["CashEquivalents"] + dfCombine["TotalLiab"])
        elif type == "LYR":
            dfCombine[factorName] = dfCombine["OperatingProfit_LYR"] / (dfCombine["Cap_Report_LYR"] - dfCombine["CashEquivalents_LYR"] + dfCombine["TotalLiab_LYR"])
    #
    return dfCombine


def PriceNetCFCallBack(dfCombine, factorName, params={}):
    #
    type = "_" + params["Type"]

    #
    r = False
    if "Reciprocal" in params:
        r = params["Reciprocal"]

    #
    if r:
        dfCombine[factorName] = (dfCombine["OperatingCashFlow" + type] +
                                 dfCombine["FinancingCashFlow" + type] +
                                 dfCombine["InvestingCashFlow" + type]) / dfCombine["Cap"]
    #
    else:
        dfCombine[factorName] = dfCombine["Cap"] / (dfCombine["OperatingCashFlow" + type] +
                                                    dfCombine["FinancingCashFlow" + type] +
                                                    dfCombine["InvestingCashFlow" + type])
    #
    return dfCombine


def CalcValuatioFactor(database, instruments, datetime1, datetime2, update=False):

    # ---Last File---
    # PB
    CalcPriceFundamentalCombinedFactor(database, instruments, datetime1, datetime2,
                                       factorName="PB_LF",
                                       fields=["TotalEquity"],
                                       callback=SimpleValuationCallBack,
                                       params={"Value": "TotalEquity"},
                                       # stratIndex=2427,
                                       update=update)

    # BookToMarket / BTOP
    CalcPriceFundamentalCombinedFactor(database, instruments, datetime1, datetime2,
                                       factorName="BookToMarket",
                                       fields=["TotalEquity"],
                                       callback=SimpleValuationReverseCallBack,
                                       params={"Value": "TotalEquity"},
                                       update=update)

    # ---LYR / Last Year (Static) DataSeries---
    # --- TTM Rolling / Trailing Twelve Months---
    for type in ["LYR", "TTM"]:
        #
        for earning in ["OperatingProfit", "OperatingProfit1", "NetIncome2"]:
            CalcPriceFundamentalCombinedFactor(database, instruments, datetime1, datetime2,
                                               factorName="PE_" + earning + "_" + type,
                                               quarterlyFields=[earning],
                                               callback=SimpleValuationCallBack,
                                               params={"Value": earning, "Type": type},
                                               update=update)

    for type in ["LYR", "TTM"]:
        # Price / Sales
        CalcPriceFundamentalCombinedFactor(database, instruments, datetime1, datetime2,
                                           factorName="PS_" + type,
                                           quarterlyFields=["Sales"],
                                           callback=SimpleValuationCallBack,
                                           params={"Value": "Sales", "Type": type},
                                           update=update)
        # Price / CF
        CalcPriceFundamentalCombinedFactor(database, instruments, datetime1, datetime2,
                                           factorName="PCF_" + type,
                                           quarterlyFields=["OperatingCashFlow"],
                                           callback=SimpleValuationCallBack,
                                           params={"Value": "OperatingCashFlow", "Type": type},
                                           update=update)
        # CF / Price -R
        CalcPriceFundamentalCombinedFactor(database, instruments, datetime1, datetime2,
                                           factorName="PCF_R_" + type,
                                           quarterlyFields=["OperatingCashFlow"],
                                           callback=SimpleValuationCallBack,
                                           params={"Value": "OperatingCashFlow", "Type": type, "Reciprocal": True},
                                           update=update)
        # Price / FCF
        CalcPriceFundamentalCombinedFactor(database, instruments, datetime1, datetime2,
                                           factorName="PFCF_" + type,
                                           quarterlyFields=["FreeCashFlow"],
                                           callback=SimpleValuationCallBack,
                                           params={"Value": "FreeCashFlow", "Type": type},
                                           update=update)
        # Price / Net CF
        CalcPriceFundamentalCombinedFactor(database, instruments, datetime1, datetime2,
                                           factorName="PNetCF_" + type,
                                           quarterlyFields=["OperatingCashFlow", "FinancingCashFlow", "InvestingCashFlow"],
                                           callback=PriceNetCFCallBack,
                                           params={"Type": type},
                                           update=update)
        # Net CF / Price -R
        CalcPriceFundamentalCombinedFactor(database, instruments, datetime1, datetime2,
                                           factorName="PNetCF_R_" + type,
                                           quarterlyFields=["OperatingCashFlow", "FinancingCashFlow", "InvestingCashFlow"],
                                           callback=PriceNetCFCallBack,
                                           params={"Type": type, "Reciprocal": True},
                                           update=update)
        # Enterprise Multiple - R
        CalcPriceFundamentalCombinedFactor(database, instruments, datetime1, datetime2,
                                          factorName="EnterpriseMultiple_R_" + type,
                                          fields=["CashEquivalents", "TotalLiab", "Close", "TotalShares"],
                                          quarterlyFields=["OperatingProfit"], #EBIT
                                          callback=EBITTEVCallBack,
                                          params={"Type": type, "Reciprocal": True},
                                          update=update)

        # Earning Yield
        CalcPriceFundamentalCombinedFactor(database, instruments, datetime1, datetime2,
                                           factorName="EarningToPrice_" + type,
                                           quarterlyFields=["NetIncome2"],
                                           callback=SimpleValuationCallBack,
                                           params={"Value": "NetIncome2", "Type": type, "Reciprocal": True},
                                           update=update)
        # Cash Earning Yield
        CalcPriceFundamentalCombinedFactor(database, instruments, datetime1, datetime2,
                                           factorName="CashEarningToPrice_" + type,
                                           quarterlyFields=["OperatingCashFlow"],
                                           callback=SimpleValuationCallBack,
                                           params={"Value": "OperatingCashFlow", "Type": type, "Reciprocal": True},
                                           update=update)
        pass


def CalcValuationRanking(database, datetime1, datetime2):
    CalcRanking2(database, "PriceEarningOperatingProfitTTM", datetime1, datetime2)


def CalcValuationHistogram(database, datetime1, datetime2):
    #
    filter = {}
    #filter = {"Market": ""}

    #
    instruments = database.find("Instruments", "Stock", query=filter)

    instruments2 = []
    for instrument in instruments:
        instruments2.append(instrument)
        if len(instruments2) >= 10:
            break
    instruments = instruments2

    #
    factorName = "PriceEarningOperatingProfitTTM"
    #

    #dfData = IO.LoadMultiInstrumentsFactorsAsDataFrame(database, factorName, datetime1, datetime2, instruments)

    dfData = IO.LoadFactorsProfileAsDataFrame(database, datetime2, instruments, [factorName])
    print(dfData)
    max = dfData[factorName].max()
    min = dfData[factorName].max()

    #for i in len(dfData):
    #    for j in range(0, int(max), 10):
    #        print(j)
    #kkwood = 0


#---计算Growth Factor---
def CalcValuatioFactorOld(database, instruments, datetime1, datetime2):
    count = 0
    start = False
    for instrument in instruments:
        count = count + 1
        symbol = instrument["Symbol"]
        #
        #if symbol == "000030.SZ":
        #    start = True
        #if symbol == "000166.SZ":
        #    start = True
        #if symbol == "000333.SZ":
        #    start = True
        #if symbol == "000506.SZ": # cashflow = 0
        #    start = True
        # if symbol == "000620.SZ": # no total equity no OperatingProfit1, ReleaseDate 和 ReportDate Gap过大 ，2009年四季度无法计算TTm（missing 2008Report3）
        #    start = True
        # if symbol == "000719.SZ": # Total Equity = 0 @2009-12-31
        #    start = True
        # if symbol == "000748.SZ": # Missing Report @2016-12-31
        #    start = True
        # if symbol == "600823.SH": # Missing FreeCashFlow @2006-03-31 (Fixed)
        #    start = True
        if symbol == "002879.SZ": #
            start = True
        if not start:
            continue

        # ---Create DataSeries---
        dataSeriesCap = DataSeries.DataSeries(symbol + "_Cap_Factor")
        dataSeriesPriceBook = DataSeries.DataSeries(symbol + "_PriceBookLF_Factor")
        dataSeriesBookToMarket = DataSeries.DataSeries(symbol + "_BookToMarket_Factor")

        # ---LYR (Static) DataSeries---
        dataSeriesPriceEarningOperatingProfit_LYR = DataSeries.DataSeries(symbol + "_PriceEarningOperatingProfitLYR_Factor")
        dataSeriesPriceEarningTotalIncome_LYR = DataSeries.DataSeries(symbol + "_PriceEarningTotalIncomeLYR_Factor")
        dataSeriesPriceEarningNetIncome_LYR = DataSeries.DataSeries(symbol + "_PriceEarningNetIncomeLYR_Factor")
        #
        dataSeriesPriceSales_LYR = DataSeries.DataSeries(symbol + "_PriceSalesLYR_Factor")
        dataSeriesPriceCashFlow_LYR = DataSeries.DataSeries(symbol + "_PriceCashFlowLYR_Factor")
        dataSeriesPriceFreeCashFlow_LYR = DataSeries.DataSeries(symbol + "_PriceFreeCashFlowLYR_Factor")

        # --- TTM Rolling ---
        dataSeriesPriceEarningOperatingProfit_TTM = DataSeries.DataSeries(symbol + "_PriceEarningOperatingProfitTTM_Factor")
        dataSeriesPriceEarningTotalIncome_TTM = DataSeries.DataSeries(symbol + "_PriceEarningTotalIncomeTTM_Factor")
        dataSeriesPriceEarningNetIncome_TTM = DataSeries.DataSeries(symbol + "_PriceEarningNetIncomeTTM_Factor")
        #
        dataSeriesPriceSales_TTM = DataSeries.DataSeries(symbol + "_PriceSalesTTM_Factor")
        dataSeriesPriceCashFlow_TTM = DataSeries.DataSeries(symbol + "_PriceCashFlowTTM_Factor")
        dataSeriesPriceFreeCashFlow_TTM = DataSeries.DataSeries(symbol + "_PriceFreeCashFlowTTM_Factor")

        #
        filter = {}
        sort = [("StdReportDate", pymongo.ASCENDING)]
        # ---读取全部Statements---
        # statements = database.findWithFilter("Fundamental", symbol + "_Fundamental", filter, sort)
        statements = database.getDataSeries(symbol + "_Fundamental")
        quotes = database.getDataSeries(symbol + "_Time_86400_Bar", "Stock", datetime1, datetime2)

        # ---整理出所有的年报---
        annualStatements = []
        for statement in statements.items:
            if statement["Period"] == 4:
                annualStatements.append(statement)

        # ---DateTime Range---
        minimalDT = datetime.datetime(1980, 1, 1)
        minimalDT = Gadget.ToUTCDateTime(minimalDT)
        #
        ipoDT = instrument["IPO_Date"]
        ipoDT = Gadget.ToUTCDateTime(ipoDT)
        #
        beginDT = datetime1
        if ipoDT > datetime1:
            beginDT = ipoDT

        #
        quote = None
        lastStatement = None
        # datetimes = Gadget.GenerateEndDayofMonth(datetime1, datetime2)
        datetimes = Gadget.GenerateMonthDates(beginDT, datetime2)

        # --- Loop Dates (Beginning day of Month) ---
        for dt in datetimes:
            # print("### " + str(dt) + " ###")
            # Time Stop point
            if dt.year == 2017 and dt.month == 10 and dt.day == 31:
                kkwood = 0

            # ---Finds the nearest quote and statement---
            quote = quotes.Get(dt, "Previous")
            if quote is None:
                print(symbol + " No Quote @BacktestDate " + str(dt))
                continue
            #
            # lastStatement = Gadget.Find(statements, dt, "Previous")  # Last File
            lastStatement = statements.Get(dt, "Previous")  # Last File
            if lastStatement is None:
                print(symbol + " No LastStatement @BacktestDate " + str(dt))
                continue

            #
            reportDate = lastStatement["ReportDate"]
            releaseDate = lastStatement["ReleaseDate"]
            quoteDT = quote["StdDateTime"]
            reportDT = lastStatement["StdReportDate"]
            releaseDT = lastStatement["StdDateTime"]
            #
            if releaseDT < minimalDT:
                print(symbol + " ReleaseDate Error, May no Statement Exist @BacktestDate " + str(dt))
                continue
            #
            backtestYear = dt.year
            lastReleaseYear = releaseDT.year

            # --- 关键事件点，报告期---
            # --- Fix Special Case, Release Data & Report Data 差距达到两年（000620.SZ） ---
            # --- 从所能得到的最近一次报告往前推 ---
            lastYearDateTime1 = datetime.datetime(lastReleaseYear - 1, 3, 31)  # 去年一季报 基于 current backtest date
            lastYearDateTime2 = datetime.datetime(lastReleaseYear - 1, 6, 30)  # 去年半年报
            lastYearDateTime3 = datetime.datetime(lastReleaseYear - 1, 9, 30)  # 去年三季报
            lastYearDateTime4 = datetime.datetime(lastReleaseYear - 1, 12, 31)  # 去年年报
            lastYearDateTime1 = Gadget.ToUTCDateTime(lastYearDateTime1)
            lastYearDateTime2 = Gadget.ToUTCDateTime(lastYearDateTime2)
            lastYearDateTime3 = Gadget.ToUTCDateTime(lastYearDateTime3)
            lastYearDateTime4 = Gadget.ToUTCDateTime(lastYearDateTime4)
            #
            # thisYearDateTime1 = datetime.datetime(reportYear, 3, 31)  # 今年一季报
            # thisYearDateTime2 = datetime.datetime(reportYear, 6, 30)  # 今年半年报
            # thisYearDateTime3 = datetime.datetime(reportYear, 9, 30)  # 今年三季报
            # thisYearDateTime1 = Gadget.ToUTCDateTime(thisYearDateTime1)
            # thisYearDateTime2 = Gadget.ToUTCDateTime(thisYearDateTime2)
            # thisYearDateTime3 = Gadget.ToUTCDateTime(thisYearDateTime3)

            # --- 已经通过 lastReleaseYear 确定了需要知道的“去年” ---
            # --- 通过报告期，不是发布日期，找到准确的报告---
            lastYearStatement1 = Gadget.Find(statements.items, lastYearDateTime1, "Exact", datetimeField="StdReportDate")  # 去年一季报
            lastYearStatement2 = Gadget.Find(statements.items, lastYearDateTime2, "Exact", datetimeField="StdReportDate")  # 去年半年报
            lastYearStatement3 = Gadget.Find(statements.items, lastYearDateTime3, "Exact", datetimeField="StdReportDate")  # 去年三季报
            lastYearStatement4 = Gadget.Find(statements.items, lastYearDateTime4, "Exact", datetimeField="StdReportDate")  # 去年年报
            #
            # thisYearStatement1 = Gadget.Find(statements.items, thisYearDateTime1, "Exact", datetimeField="StdReportDate")  # 今年一季报
            # thisYearStatement2 = Gadget.Find(statements.items, thisYearDateTime2, "Exact", datetimeField="StdReportDate")  # 今年半年报
            # thisYearStatement3 = Gadget.Find(statements.items, thisYearDateTime3, "Exact", datetimeField="StdReportDate")  # 今年三季报

            # ---Cache to Check Null Values---
            previousStatements = []
            previousStatements.append(lastYearStatement1)
            previousStatements.append(lastYearStatement2)
            previousStatements.append(lastYearStatement3)
            previousStatements.append(lastYearStatement4)

            # ---最近的一次去年年报---
            # ---如果去年的不存在，没准还可以取到前年的---
            # lastYearStatement = Gadget.Find(annualStatements.items, dt, "Exact", datetimeField="StdDateTime")
            # ---如果去年的不存在，就不用算了---
            lastYearStatement = lastYearStatement4
            if lastStatement["Period"] == 4:
                lastYearStatement = lastStatement

            #
            document = {}
            document["Symbol"] = symbol
            document["StdDateTime"] = quote["StdDateTime"]
            #
            capDoc = copy.deepcopy(document)
            bookToMarketDoc = copy.deepcopy(document)
            priceBookDoc = copy.deepcopy(document)
            #
            priceEarningOperatingProfitDoc_LYR = copy.deepcopy(document)
            priceEarningTotalIncomeDoc_LYR = copy.deepcopy(document)
            priceEarningNetIncomeDoc_LYR = copy.deepcopy(document)
            #
            priceSalesDoc_LYR = copy.deepcopy(document)
            priceCashFlowDoc_LYR = copy.deepcopy(document)
            priceFreeCashFlowDoc_LYR = copy.deepcopy(document)
            #
            priceEarningOperatingProfitDoc_TTM = copy.deepcopy(document)
            priceEarningTotalIncomeDoc_TTM = copy.deepcopy(document)
            priceEarningNetIncomeDoc_TTM = copy.deepcopy(document)
            #
            priceSalesDoc_TTM = copy.deepcopy(document)
            priceCashFlowDoc_TTM = copy.deepcopy(document)
            priceFreeCashFlowDoc_TTM = copy.deepcopy(document)

            #
            completeCalcLF = False
            completeCalcLYR = False
            completeToCalcTTM = False
            #
            lastStatementReady = True
            lastYearStatementReady = True
            ttmDependentStatementReady = True
            #
            try:
                cap = quote["Close"] * quote["Values"]["TotalShares"]
                capDoc["Value"] = cap

                # --- Check Dependent Statement ---
                if lastStatement is None or "Values" not in lastStatement:
                    lastStatementReady = False
                #
                elif "OperatingProfit1" not in lastStatement["Values"]:
                    lastStatementReady = False
                elif "TotalEquity" not in lastStatement["Values"]:
                    lastStatementReady = False

                # --- Calculation for LF (Last File)---
                if lastStatementReady:
                    book = lastStatement["Values"]["TotalEquity"]
                    bookToMarketDoc["Value"] = book / cap
                    #
                    if book == 0:
                        priceBookDoc["Value"] = 0
                    else:
                        priceBookDoc["Value"] = cap / book
                    # priceBookDoc["Value"] = cap / book
                    #
                    completeCalcLF = True

                else:
                    print(symbol + " CalcValuatioFactor::Can't Calc LF Factor, Missing Last-(Recent)-Statement " + " @BacktestDate " + str(dt) + " @ReportDate " + reportDate)


                # --- Recent Last Year: LYR---
                priceEarningOperatingProfit_LYR = 0
                priceEarningTotalIncome_LYR = 0
                priceEarningNetIncome_LYR = 0
                priceSales_LYR = 0
                priceCashFlow_LYR = 0
                priceFreeCashFlow_LYR = 0

                # --- Check Dependent Statement ---
                if lastYearStatement is None or "Values" not in lastYearStatement:
                    lastYearStatementReady = False

                # --- Calculation for LYR ---
                if lastYearStatementReady:
                    priceEarningOperatingProfit_LYR = cap / lastYearStatement["Values"]["OperatingProfit1"] # Operating Profit
                    priceEarningTotalIncome_LYR = cap / lastYearStatement["Values"]["TotalIncome"]  # Plus Investment / Extrardinary Item
                    priceEarningNetIncome_LYR = cap / lastYearStatement["Values"]["NetIncome2"]  # -Tax, +Minority Income
                    #
                    if lastYearStatement["Values"]["Sales"] != 0:
                        priceSales_LYR = cap / lastYearStatement["Values"]["Sales"]

                    if lastYearStatement["Values"]["OperatingCashFlow"] != 0:
                        priceCashFlow_LYR = cap / lastYearStatement["Values"]["OperatingCashFlow"]

                    if lastYearStatement["Values"]["FreeCashFlow"] != 0:
                        priceFreeCashFlow_LYR = cap / lastYearStatement["Values"]["FreeCashFlow"]
                    #
                    priceEarningOperatingProfitDoc_LYR["Value"] = priceEarningOperatingProfit_LYR
                    priceEarningTotalIncomeDoc_LYR["Value"] = priceEarningTotalIncome_LYR
                    priceEarningNetIncomeDoc_LYR["Value"] = priceEarningNetIncome_LYR
                    priceSalesDoc_LYR["Value"] = priceSales_LYR
                    priceCashFlowDoc_LYR["Value"] = priceCashFlow_LYR
                    priceFreeCashFlowDoc_LYR["Value"] = priceFreeCashFlow_LYR
                    #
                    completeCalcLYR = True
                else:
                    print(symbol + " CalcValuatioFactor::Can't Calc LYR Factor, Missing Last-Year-Statement " + " @BacktestDate " + str(dt) + " @ReportDate " + reportDate)

                # ---Calc TTM---
                priceEarningOperatingProfit_TTM = 0
                priceEarningTotalIncome_TTM = 0
                priceEarningNetIncome_TTM = 0
                priceSales_TTM = 0
                priceCashFlow_TTM = 0
                priceFreeCashFlow_TTM = 0

                # --- Check Dependent Statement ---
                if lastStatement["Period"] == 1 or lastStatement["Period"] == 2 or lastStatement["Period"] == 3:
                    period = lastStatement["Period"]
                    index = period - 1
                    #
                    if lastYearStatement4 is None or "Values" not in lastYearStatement4:
                        ttmDependentStatementReady = False
                    #
                    elif previousStatements[index] is None or "Values" not in previousStatements[index]:
                        ttmDependentStatementReady = False
                    #
                    elif "OperatingProfit1" not in previousStatements[index]["Values"]:
                        ttmDependentStatementReady = False
                    #
                    elif "OperatingProfit1" not in lastYearStatement4["Values"]:
                        ttmDependentStatementReady = False

                # --- Calculation for TTM ---
                if ttmDependentStatementReady:
                    #
                    if lastStatement["Period"] == 1:
                        # 2-3-4-1
                        # ---Operating Profit---
                        priceEarningOperatingProfit_TTM = lastStatement["Values"]["OperatingProfit1"] + lastYearStatement4["Values"]["OperatingProfit1"] - lastYearStatement1["Values"]["OperatingProfit1"]
                        # --- Plus Investment / Extrardinary Item---
                        priceEarningTotalIncome_TTM = lastStatement["Values"]["TotalIncome"] + lastYearStatement4["Values"]["TotalIncome"] - lastYearStatement1["Values"]["TotalIncome"]
                        # --- substract Tax, plus Minority Income ---
                        priceEarningNetIncome_TTM = lastStatement["Values"]["NetIncome2"] + lastYearStatement4["Values"]["NetIncome2"] - lastYearStatement1["Values"]["NetIncome2"]
                        #
                        priceSales_TTM = lastStatement["Values"]["Sales"] + lastYearStatement4["Values"]["Sales"] - lastYearStatement1["Values"]["Sales"]
                        priceCashFlow_TTM = lastStatement["Values"]["OperatingCashFlow"] + lastYearStatement4["Values"]["OperatingCashFlow"] - lastYearStatement1["Values"]["OperatingCashFlow"]
                        priceFreeCashFlow_TTM = lastStatement["Values"]["FreeCashFlow"] + lastYearStatement4["Values"]["FreeCashFlow"] - lastYearStatement1["Values"]["FreeCashFlow"]

                    elif lastStatement["Period"] == 2:
                        # 3-4-1-2
                        #
                        priceEarningOperatingProfit_TTM = lastStatement["Values"]["OperatingProfit1"] + lastYearStatement4["Values"]["OperatingProfit1"] - lastYearStatement2["Values"]["OperatingProfit1"]
                        priceEarningTotalIncome_TTM = lastStatement["Values"]["TotalIncome"] + lastYearStatement4["Values"]["TotalIncome"] - lastYearStatement2["Values"]["TotalIncome"]
                        priceEarningNetIncome_TTM = lastStatement["Values"]["NetIncome2"] + lastYearStatement4["Values"]["NetIncome2"] - lastYearStatement2["Values"]["NetIncome2"]
                        #
                        priceSales_TTM = lastStatement["Values"]["Sales"] + lastYearStatement4["Values"]["Sales"] - lastYearStatement2["Values"]["Sales"]
                        priceCashFlow_TTM = lastStatement["Values"]["OperatingCashFlow"] + lastYearStatement4["Values"]["OperatingCashFlow"] - lastYearStatement2["Values"]["OperatingCashFlow"]
                        priceFreeCashFlow_TTM = lastStatement["Values"]["FreeCashFlow"] + lastYearStatement4["Values"]["FreeCashFlow"] - lastYearStatement2["Values"]["FreeCashFlow"]

                    elif lastStatement["Period"] == 3:
                        # 4-1-2-3
                        #
                        priceEarningOperatingProfit_TTM = lastStatement["Values"]["OperatingProfit1"] + lastYearStatement4["Values"]["OperatingProfit1"] - lastYearStatement3["Values"]["OperatingProfit1"]
                        priceEarningTotalIncome_TTM = lastStatement["Values"]["TotalIncome"] + lastYearStatement4["Values"]["TotalIncome"] - lastYearStatement3["Values"]["TotalIncome"]
                        priceEarningNetIncome_TTM = lastStatement["Values"]["NetIncome2"] + lastYearStatement4["Values"]["NetIncome2"] - lastYearStatement3["Values"]["NetIncome2"]
                        #
                        sale1 = lastStatement["Values"]["Sales"]
                        sale2 = lastYearStatement4["Values"]["Sales"]
                        sale3 = lastYearStatement3["Values"]["Sales"]
                        priceSales_TTM = lastStatement["Values"]["Sales"] + lastYearStatement4["Values"]["Sales"] - lastYearStatement3["Values"]["Sales"]
                        priceCashFlow_TTM = lastStatement["Values"]["OperatingCashFlow"] + lastYearStatement4["Values"]["OperatingCashFlow"] - lastYearStatement3["Values"]["OperatingCashFlow"]
                        priceFreeCashFlow_TTM = lastStatement["Values"]["FreeCashFlow"] + lastYearStatement4["Values"]["FreeCashFlow"] - lastYearStatement3["Values"]["FreeCashFlow"]

                    elif lastStatement["Period"] == 4:
                        # 1-2-3-4
                        priceEarningOperatingProfit_TTM = lastStatement["Values"]["OperatingProfit1"]
                        priceEarningTotalIncome_TTM = lastStatement["Values"]["TotalIncome"]
                        priceEarningNetIncome_TTM = lastStatement["Values"]["NetIncome2"]
                        #
                        priceSales_TTM = lastStatement["Values"]["Sales"]
                        priceCashFlow_TTM = lastStatement["Values"]["OperatingCashFlow"]
                        priceFreeCashFlow_TTM = lastStatement["Values"]["FreeCashFlow"]

                    else:
                        print(symbol + " CalcValuatioFactor::UnknownReportPeriod @ReportDate " + reportDate)
                        ttmDependentStatementReady = False

                # --- Calculation for TTM ---
                if ttmDependentStatementReady:
                    #
                    priceEarningOperatingProfitDoc_TTM["Value"] = cap / priceEarningOperatingProfit_TTM  # Operating Profit
                    priceEarningTotalIncomeDoc_TTM["Value"] = cap / priceEarningTotalIncome_TTM  # Plus Investment / Extrardinary Item
                    priceEarningNetIncomeDoc_TTM["Value"] = cap / priceEarningNetIncome_TTM  # -Tax, +Minority Income
                    #
                    if priceSales_TTM != 0:
                        priceSalesDoc_TTM["Value"] = cap / priceSales_TTM
                    priceCashFlowDoc_TTM["Value"] = cap / priceCashFlow_TTM
                    priceFreeCashFlowDoc_TTM["Value"] = cap / priceFreeCashFlow_TTM
                    completeToCalcTTM = True
                else:
                    print(symbol + " CalcValuatioFactor::Can't Calc TTM Factor, Missing Some Previous Dependent Statements " + " @BacktestDate " + str(dt) + " @ReportDate " + reportDate)

            except ZeroDivisionError as e:
                print(symbol + " CalcValuatioFactor::Calculation ZeroDivisionError " + str(e) + " @BacktestDate " + str(dt) + " @ReportDate " + reportDate)
            except KeyError as e:
                print(symbol + " CalcValuatioFactor::Calculation KeyError " + str(e) + " @BacktestDate " + str(dt) + " @ReportDate " + reportDate)
            except Exception as e:
                print(symbol + " CalcValuatioFactor::Calculation Serious Error " + " @BacktestDate " + str(dt) + " @ReportDate " + reportDate)
                print(e)
                kkwood = 0

            # --- Add document to DataSeries ---
            dataSeriesCap.Add(capDoc)
            #
            if completeCalcLF:
                dataSeriesBookToMarket.Add(bookToMarketDoc)
                dataSeriesPriceBook.Add(priceBookDoc)
            #
            if completeCalcLYR:
                dataSeriesPriceEarningOperatingProfit_LYR.Add(priceEarningOperatingProfitDoc_LYR)
                dataSeriesPriceEarningTotalIncome_LYR.Add(priceEarningTotalIncomeDoc_LYR)
                dataSeriesPriceEarningNetIncome_LYR.Add(priceEarningNetIncomeDoc_LYR)
                #
                dataSeriesPriceSales_LYR.Add(priceSalesDoc_LYR)
                dataSeriesPriceCashFlow_LYR.Add(priceCashFlowDoc_LYR)
                dataSeriesPriceFreeCashFlow_LYR.Add(priceFreeCashFlowDoc_LYR)
            #
            if completeToCalcTTM:
                dataSeriesPriceEarningOperatingProfit_TTM.Add(priceEarningOperatingProfitDoc_TTM)
                dataSeriesPriceEarningTotalIncome_TTM.Add(priceEarningTotalIncomeDoc_TTM)
                dataSeriesPriceEarningNetIncome_TTM.Add(priceEarningNetIncomeDoc_TTM)
                #
                dataSeriesPriceSales_TTM.Add(priceSalesDoc_TTM)
                dataSeriesPriceCashFlow_TTM.Add(priceCashFlowDoc_TTM)
                dataSeriesPriceFreeCashFlow_TTM.Add(priceFreeCashFlowDoc_TTM)

        # ---Loop DateTimes---

        # ---Save to database---
        database.saveDataSeries(dataSeriesCap)
        database.saveDataSeries(dataSeriesBookToMarket)
        database.saveDataSeries(dataSeriesPriceBook)
        #
        database.saveDataSeries(dataSeriesPriceEarningOperatingProfit_LYR)
        database.saveDataSeries(dataSeriesPriceEarningTotalIncome_LYR)
        database.saveDataSeries(dataSeriesPriceEarningNetIncome_LYR)
        #
        database.saveDataSeries(dataSeriesPriceSales_LYR)
        database.saveDataSeries(dataSeriesPriceCashFlow_LYR)
        database.saveDataSeries(dataSeriesPriceFreeCashFlow_LYR)
        #
        database.saveDataSeries(dataSeriesPriceEarningOperatingProfit_TTM)
        database.saveDataSeries(dataSeriesPriceEarningTotalIncome_TTM)
        database.saveDataSeries(dataSeriesPriceEarningNetIncome_TTM)
        #
        database.saveDataSeries(dataSeriesPriceSales_TTM)
        database.saveDataSeries(dataSeriesPriceCashFlow_TTM)
        database.saveDataSeries(dataSeriesPriceFreeCashFlow_TTM)

        # ---Finished One Symbol---
        print("Calculated Valuation Factor::Finished to Calc Factors " + symbol + " count:" + str(count) + " @CurrentTime " + str(datetime.datetime.now()))

    # ---Loop Instrument---


# based on pandas
def CalcValuatioFactorOld(database, instruments, datetime1, datetime2, update=False):
    #
    outputFolder = "D:/Data/CalcValuationFactor/"
    #
    count = 0
    for instrument in instruments:
        symbol = instrument["Symbol"]
        count += 1
        #if count <= 190: #000409.SZ
        #  continue
        # if symbol != "300580.SZ":
        #    continue

        # ---Rebuild DateTime Range ---
        beginDate = datetime1
        ipoDate = instrument["IPO_Date"]
        ipoDate = Gadget.ToUTCDateTime(ipoDate)
        if ipoDate > datetime1:
            beginDate = ipoDate

        endDate = datetime2
        if "Delist_Date" in instrument:
            delistDate = instrument["Delist_Date"]
            delistDate = Gadget.ToUTCDateTime(delistDate)
            if delistDate < datetime2:
                endDate = delistDate

        # ---Create DataSeries---
        dataSeriesByName = {}

        # ---Last File---
        dataSeriesByName["Cap"] = DataSeries.DataSeries(symbol + "_Cap_Factor")
        dataSeriesByName["PriceBookLF"] = DataSeries.DataSeries(symbol + "_PriceBookLF_Factor")
        dataSeriesByName["BookToMarket"] = DataSeries.DataSeries(symbol + "_BookToMarket_Factor")

        # ---LYR (Static) DataSeries---
        dataSeriesByName["PriceEarningOperatingProfitLYR"] = DataSeries.DataSeries(symbol + "_PriceEarningOperatingProfitLYR_Factor")
        dataSeriesByName["PriceEarningTotalIncomeLYR"] = DataSeries.DataSeries(symbol + "_PriceEarningTotalIncomeLYR_Factor")
        dataSeriesByName["PriceEarningNetIncomeLYR"] = DataSeries.DataSeries(symbol + "_PriceEarningNetIncomeLYR_Factor")
        dataSeriesByName["PriceSalesLYR"] = DataSeries.DataSeries(symbol + "_PriceSalesLYR_Factor")
        dataSeriesByName["PriceCashFlowLYR"] = DataSeries.DataSeries(symbol + "_PriceCashFlowLYR_Factor")
        dataSeriesByName["PriceFreeCashFlowLYR"] = DataSeries.DataSeries(symbol + "_PriceFreeCashFlowLYR_Factor")
        dataSeriesByName["EBITTEVLYR"] = DataSeries.DataSeries(symbol + "_EBITTEVLYR_Factor")

        # --- TTM Rolling ---
        dataSeriesByName["PriceEarningOperatingProfitTTM"] = DataSeries.DataSeries(symbol + "_PriceEarningOperatingProfitTTM_Factor")
        dataSeriesByName["PriceEarningTotalIncomeTTM"] = DataSeries.DataSeries(symbol + "_PriceEarningTotalIncomeTTM_Factor")
        dataSeriesByName["PriceEarningNetIncomeTTM"] = DataSeries.DataSeries(symbol + "_PriceEarningNetIncomeTTM_Factor")
        dataSeriesByName["PriceSalesTTM"] = DataSeries.DataSeries(symbol + "_PriceSalesTTM_Factor")
        dataSeriesByName["PriceCashFlowTTM"] = DataSeries.DataSeries(symbol + "_PriceCashFlowTTM_Factor")
        dataSeriesByName["PriceFreeCashFlowTTM"] = DataSeries.DataSeries(symbol + "_PriceFreeCashFlowTTM_Factor")
        dataSeriesByName["EBITTEVTTM"] = DataSeries.DataSeries(symbol + "_EBITTEVTTM_Factor")


        # ---Load Price into DataFrame---
        dfPrice = IO.LoadBarsAsDataFrame(database, symbol, beginDate, endDate, fields=["Close", "TotalShares"])

        # ---Correct DateTime---
        # ---Convert String DateTime to Std DateTime---
        for index in dfPrice.index:
            newDatetime = pd.to_datetime(dfPrice.at[index, "DateTime"])
            dfPrice.at[index, "DateTime"] = datetime.datetime(newDatetime.year, newDatetime.month, newDatetime.day)
        dfPrice['DateTime'] = pd.to_datetime(dfPrice['DateTime'])
        #print(dfPrice)
        #print(dfPrice.info())


        # ---Load Fundamental into DataFrame---
        fields = []
        fields.append("TotalEquity")
        fields.append("CashEquivalents")
        fields.append("TotalLiab")
        fields.append("OperatingProfit1")
        fields.append("TotalIncome")
        fields.append("NetIncome2")
        fields.append("Sales")
        fields.append("OperatingCashFlow")
        fields.append("FreeCashFlow")
        fields.append("EBIT")
        dfFundamental = IO.LoadFundamentalsAsDataFrame(database, symbol, datetime1=None, datetime2=datetime2, fields=fields)
        #print(dfFundamental)
        #dfFundamental.to_csv(outputFolder + "dfFundamental_" + symbol + ".csv")

        # --- Process LYR TTM---
        fields_LYRs = []
        fields_TTMs = []
        fields = []
        fields.append("OperatingProfit1")
        fields.append("TotalIncome")
        fields.append("NetIncome2")
        fields.append("Sales")
        fields.append("OperatingCashFlow")
        fields.append("FreeCashFlow")
        fields.append("EBIT")
        for field in fields:
            # ---LYR---
            CalculateLYR2(database, dfFundamental, field)
            # ---TTM---
            CalculateTTM2(database, dfFundamental, field)
            #
            fields_LYRs.append(field + "_LYR")
            fields_TTMs.append(field + "_TTM")

        #print(dfFundamental)
        #dfFundamental.to_csv(outputFolder + "Fundamental_" + symbol + ".csv")

        # ---Merge 时， Release Date 可能不是交易日----
        # ---所以最后拼接出来的总日期有可能混入非交易日---
        # 统一DateTime，以便排序---
        dfFundamental["DateTime"] = dfFundamental["ReleaseDate"]
        dfCombine = pd.merge(dfPrice, dfFundamental, on='DateTime', how='outer')
        dfCombine.sort_values("DateTime", inplace=True)
        dfCombine.fillna(method='ffill', inplace=True)
        #print(dfCombine)
        dfCombine.to_csv(outputFolder + "dfFundamental+Price_" + symbol + ".csv")

        # ---Calculation---
        # ---Calculation Cap Factor---
        dfCombine["Cap"] = dfCombine["Close"] * dfCombine["TotalShares"]

        # ---Calculation Last File Factor ---
        dfCombine["PriceBookLF"] = dfCombine["Cap"] / dfCombine["TotalEquity"]
        dfCombine["BookToMarket"] = dfCombine["TotalEquity"] / dfCombine["Cap"]

        # ---Calculation LYR  (Static) Factor---
        dfCombine["PriceEarningOperatingProfitLYR"] = dfCombine["Cap"] / dfCombine["OperatingProfit1_LYR"]
        dfCombine["PriceEarningTotalIncomeLYR"] = dfCombine["Cap"] / dfCombine["TotalIncome_LYR"]
        dfCombine["PriceEarningNetIncomeLYR"] = dfCombine["Cap"] / dfCombine["NetIncome2_LYR"]
        dfCombine["PriceSalesLYR"] = dfCombine["Cap"] / dfCombine["Sales_LYR"]
        dfCombine["PriceCashFlowLYR"] = dfCombine["Cap"] / dfCombine["OperatingCashFlow_LYR"]
        dfCombine["PriceFreeCashFlowLYR"] = dfCombine["Cap"] / dfCombine["FreeCashFlow_LYR"]
        dfCombine["EBITTEVLYR"] = dfCombine["EBIT_LYR"] / (dfCombine["Cap"] + dfCombine["CashEquivalents"] + dfCombine["TotalLiab"])

        # ---Calculation TTM (Rolling) Factor---
        dfCombine["PriceEarningOperatingProfitTTM"] = dfCombine["Cap"] / dfCombine["OperatingProfit1_TTM"]
        dfCombine["PriceEarningTotalIncomeTTM"] = dfCombine["Cap"] / dfCombine["TotalIncome_TTM"]
        dfCombine["PriceEarningNetIncomeTTM"] = dfCombine["Cap"] / dfCombine["NetIncome2_TTM"]
        dfCombine["PriceSalesTTM"] = dfCombine["Cap"] / dfCombine["Sales_TTM"]
        dfCombine["PriceCashFlowTTM"] = dfCombine["Cap"] / dfCombine["OperatingCashFlow_TTM"]
        dfCombine["PriceFreeCashFlowTTM"] = dfCombine["Cap"] / dfCombine["FreeCashFlow_TTM"]
        dfCombine["EBITTEVTTM"] = dfCombine["EBIT_TTM"] / (dfCombine["Cap"] - dfCombine["CashEquivalents"] + dfCombine["TotalLiab"])

        dfCombine.to_csv(outputFolder + "dfCombine_" + symbol + ".csv")

        #
        bmQuotes = database.getDataSeries("000001.SH_Time_86400_Bar", datetime1=beginDate, datetime2=endDate, instrumentType="Index")
        # --- Loop Dates (Beginning day of Month) ---
        # --- 按照 TradeDate Time 1500 来记录 factor value---
        datetimes = Gadget.GenerateMonthDates(beginDate, endDate)
        for dt in datetimes:
            bmBar = bmQuotes.Get(dt)
            if bmBar == None:
                continue
            tradeDateTime = bmBar["DateTime"]
            # print("MonthDate " + str(dt) + " TradeDate " + str(tradeDateTime))
            localTradeDate = Gadget.ToDate(tradeDateTime)

            #---Find Recent Trade Date Value---
            dfTemp = dfCombine[dfCombine['DateTime'] <= localTradeDate]
            # print("Total " + str(len(dfCombine)) + " LessThenDate " + str(len(dfTemp)))

            if len(dfTemp) == 0:
                continue

            dfTemp = dfTemp.iloc[-1,:]
            # print(dfTemp)
            # print(dfTemp["DateTime"])
            # continue

            # --- Generate Document, Fill to DataSeries ---
            for name, dataSeries in dataSeriesByName.items():
                value = dfTemp[name]
                if np.isnan(value):
                    continue
                document = GenarateDocument(symbol, tradeDateTime, value)
                dataSeries.Add(document)

        # ---Save to database---
        for name, dataSeries in dataSeriesByName.items():
           database.saveDataSeries(dataSeries)

        #
        # for name, dataSeries in dataSeriesByName.items():
        #    print(name)
        #    dataSeries.Print()
        #    kkwood = 0

        # 2006-3-31 PriceBookLF
        # ---Finished One Symbol---
        print("Calculated Valuation Factor::Finished to Calc Factors " + symbol + " count:" + str(
            count) + " @CurrentTime " + str(datetime.datetime.now()))
        kkwood = 0


if __name__ == '__main__':
    # ---Connecting Database---
    from Core.Config import *
    config = Config()
    database = config.DataBase("MySQL")
    #
    # Price / FCF
    instruments = database.Find("Instruments", "Stock", {"Symbol":"000761.SZ"})
    datetime1 = datetime.datetime(2019,1,1)
    datetime2 = datetime.datetime(2019,10,1)
    CalcPriceFundamentalCombinedFactor(database, instruments, datetime1, datetime2,
                                       factorName="PFCF_" + "TTM",
                                       quarterlyFields=["FreeCashFlow"],
                                       callback=SimpleValuationCallBack,
                                       params={"Value": "FreeCashFlow", "Type": "TTM"},
                                       update=False)